# ---------------------------------------------------------------------------
# Author: Min Feng
# Date: 01/04/08
# searchs and builds datasets under the source folders,
#  and compresses datasets into zip format and moves them to repository folder.
#----------------------------------------------------------------------------

import sys
import os
import re
import tarfile
import zipfile
import configure

class buildDatabase:

    def __del__(self):
        if self.log:
            self.log.close()
        
    def __init__(self, temp_index):
        self.temp_index = temp_index
        self.conf = configure.configure()
        self.log = open(self.conf.getString('build.log'), 'a')
        
    def build(self):
        self.map = {}
        self.src_path = self.conf.getPathRepository()
        self.dis_path = self.conf.getList('build.inputs')
        self.temp_path = self.conf.getString('build.temp')
        
        if self.src_path == '':
            raise 'No source path'
        if self.temp_path == '':
            raise 'No temp path'

        print '---------------------------'
        print '>Temp path:', self.temp_path
        print '>Src path:', self.src_path
        print '>Dis paths:', self.dis_path
        print ''


        print 'Clean temp path...', 
        self.clearTempDir(self.temp_path)
        if os.path.exists(self.temp_path) == False:
            os.mkdir(self.temp_path)
        print 'ok'

##        if raw_input('Start build database? (y/n)').strip().lower() != 'y':
##            return
##
        import datetime
        _date = datetime.datetime.today()
        print _date
        print ''
        
        self.log.write('Start:' + str(_date) + '\r\n')

##        raise Exception('test')
        print 'Scan source path:'
        _b.listRepository(self.map, self.src_path, self.src_path)
    
        print 'Dath repository paths:'
        for _d in self.dis_path:
            _b.listRepository(self.map, self.src_path, _d)

        print '-' * 40
        print 'Building finished.'
        print datetime.datetime.today() - _date
        print 'Total file count', len(self.map)
        
        self.log.write('Building finished.\r\n')
        self.log.write(str(datetime.datetime.today() - _date))
        
    def clearTempDir(self, d):
        try:
            if os.path.exists(d):
                for root, dirs, files in os.walk(d, topdown=False):
                    for _name in files:
                        os.remove(os.path.join(root, _name))
                    for _name in dirs:
                        os.rmdir(os.path.join(root, _name))
                os.rmdir(d)
        except Exception, err:
            print 'Clear temp directory failed:', err
            

    def createTempDir(self):
        self.temp_index += 1
        _dir = os.path.join(self.temp_path, '%(index)07d' % {'index': self.temp_index});

        self.clearTempDir(_dir)
        os.mkdir(_dir)
        
        return _dir
        
    def listImage(self, f):
        try:
            _m = {}
            if f[-3:].lower() == 'tar':
                _tar = tarfile.open(f, "r")
                for _f in _tar:
                    if _f.isfile():
                        _m[_f.name] = _f.size
                _tar.close()
            elif str(f[-3:].lower()) == 'zip':
                _zip = zipfile.ZipFile(f, 'r')
                for _f in _zip.infolist():
                    _m[_f.filename] = _f.file_size
                _zip.close()
            else:
                raise 'unrecognized format:' + f[-3:].lower()

            return _m
        except Exception, err:
            print 'List image', f,'failed:', err
            raise        

    def listCombine(self, l1, l2):
        _keys = l1.keys()
        for _k in l2:
            if _k not in _keys:
                _keys.append(_k)

        _b = False
        _m = {}
        for _k in _keys:
            if _k in l1 and _k not in l2:
                _m[_k] = 0
            elif _k not in l1 and _k in l2:
                _m[_k] = 1
                _b = True
            else:
                if l1[_k] >= l2[_k]:
                    _m[_k] = 0
                else:
                    _m[_k] = 1
                    _b = True

        return _m, _b

    def uncompressFile(self, f, d):
        _list = []
        if f[-3:].lower() == 'tar':
            _tar = tarfile.open(f, 'r')
            for _f in _tar:
                _tar.extract(_f, d)
                _list.append(_f.name)
            _tar.close()
        elif f[-3:].lower() == 'zip':
            _zip = zipfile.ZipFile(f, 'r')
            for _f in _zip.infolist():
                _o = open(os.path.join(d, _f.filename), 'wb')
                _o.write(_zip.read(_f.filename))
                _list.append(_f.filename)

                _o.close()
            _zip.close()
        else:
            raise 'unrecognized format:' + f[-3:].lower()

        return _list

    def combineFiles(self, s, f1, f2, m):
        _dir1 = self.createTempDir()
        _dir2 = self.createTempDir()
        _dir3 = self.createTempDir()

        self.uncompressFile(f1, _dir1)
        self.uncompressFile(f2, _dir2)

##        print m
        for _k in m:
            _p = _k
            if os.path.split(_p)[0]:
                _p = _p.replace('/', '\\')
                _pp = os.path.join(_dir3, os.path.split(_p)[0])
                if os.path.exists(_pp) == False:
                    os.mkdir(_pp)
            
            if m[_k] == 0:
                os.rename(os.path.join(_dir1, _p), os.path.join(_dir3, _p))
            else:
##                print os.path.join(_dir2, _p), os.path.join(_dir3, _p)
                os.rename(os.path.join(_dir2, _p), os.path.join(_dir3, _p))

        _path, filename = os.path.split(f1)
        _file, _ext = os.path.splitext(filename)

        _name = os.path.join(s, _file + '.zip')
        _zip = zipfile.ZipFile(_name, 'w')
        for _k in m:
            _k = _k.replace('/', '\\')
            if os.path.isfile(os.path.join(_dir3, _k)):
                _zip.write(os.path.join(_dir3, _k), _k, zipfile.ZIP_DEFLATED)

        _zip.close()

        self.clearTempDir(_dir1)
        self.clearTempDir(_dir2)
        self.clearTempDir(_dir3)

        #delete the original file
        os.remove(f1)
        os.remove(f2)
        
    def reCompressFile(self, s, f):
        _dir1 = self.createTempDir()
        _list = self.uncompressFile(f, _dir1)

        _path, filename = os.path.split(f)
        _file, _ext = os.path.splitext(filename)

        _name = os.path.join(s, _file + '.zip')
        _zip = zipfile.ZipFile(_name, 'w')
        for _k in _list:
            _p = _k
            if os.path.split(_p)[0]:
                _p = _p.replace('/', '\\')
                _pp = os.path.join(_dir1, os.path.split(_p)[0])
                if os.path.exists(_pp) == False:
                    os.mkdir(_pp)

            if os.path.isfile(os.path.join(_dir1, _p)):
                _zip.write(os.path.join(_dir1, _p), _p, zipfile.ZIP_DEFLATED)

        _zip.close()
        self.clearTempDir(_dir1)

        #delete the original file
        os.remove(f)

        return _name

    def isNeedRecompress(self, p):
        if p[-3:].lower() == 'tar':
            return True
        elif p[-3:].lower() == 'zip':
            _zip = zipfile.ZipFile(p, 'r')
            _minType = 8
            for _f in _zip.infolist():
                if _f.compress_type < _minType:
                    _minType = _f.compress_type
            _zip.close()
##            print 'min type', _minType
            if _minType < 8:
                return True
            else:
                return False

    def addImage(self, m, s, n, f):
        print '    ', f, 
        self.log.write('---' + f + '\r\n')
        if n in m.keys():
            _list1 = self.listImage(m[n])
            _list2 = self.listImage(f)

##            print _list1
##            print _list2
##
            _m, _b = self.listCombine(_list1, _list2)
            if _b:
                print 'combine to', m[n], '...',
                self.log.write('combine to' + m[n] + '...')
                self.combineFiles(s, m[n], f, _m)
            else:
                os.remove(f)
            print 'removed'
            self.log.write('removed\r\n')
        else:
            if self.isNeedRecompress(f):
                print 'recompress...',
                self.log.write('recompress...')
                m[n] = self.reCompressFile(s, f)
            else:
                _f = f
                if os.path.split(f)[0] != s:
                    _f = os.path.join(s, os.path.split(f)[1])
                    os.rename(f, _f)
                    
                m[n] = _f
            print 'ok'
            self.log.write('ok\r\n')

    def listRepository(self, m, s, p):
        print ' ', p
        for _root, _dirs, _files in os.walk(p):
            for _file in _files:
                _m = re.match('(\w+\d+)\.[zip|tar]', _file)
                if _m:
                    #print '\t', _file
                    self.addImage(m, s, _m.group(1), os.path.join(_root, _file))
        

if __name__ == '__main__':
    _pos = 0
    _temp_index = 0
    while _pos < 1:
        try:
            _pos += 1
            print 'start', _pos, '...'
            _b = buildDatabase(_temp_index)
            _b.build()
            break
        except Exception, err:
            _temp_index = _b.temp_index + 1
            _b.log.write('Exception:' + str(err) + '\r\n');
            _b.log.write('Waiting for continue ' + str(_pos) + '\r\n');
            
            del _b
            
            print 'Exception:', err
            print 'Waiting for continue...'


